#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :run_movie_detail.py
# @Time      :2023/10/12 
# @Author    :CL
# @email     :1037654919@qq.com

from multiprocessing import Pool
from bs4 import BeautifulSoup
from douban import Douban
douban =Douban()
from utils import mongo_manager


def save_movie_detail(seed):
    url = seed['url']
    douban_type_list = mongo_manager('douban_type_list')
    douban_movie_detail = mongo_manager('douban_movie_detail', db='douban')
    # mongo_info = douban_type_list.findOne({'url':url})
    # if mongo_info:
    #     print(mongo_info)
    #     print(f'{url}已存在，return')
    #     # seed['movie_detail_status'] = 'success'
    #     # douban_type_list.updateOne({'_id': seed["_id"]}, seed)
    #     # douban_type_list.close()
    #     return
    datas = douban.get_movie_detail(url)
    soups = BeautifulSoup(datas, 'lxml')
    try:
        infos = soups.find('div', id='info').text.strip().split('\n')
    except:
        infos = ''
    try:
        name = soups.find('h1').text.strip().split('\n')[0]
    except:
        name = ''
    try:
        year = soups.find('h1').text.strip().split('\n')[1]
    except:
        year = ''
    # print(infos)
    print(url, name, year)
    _id = url.split('/')[-2]
    interest_sectl = soups.find('div', id='interest_sectl').text.strip().split('\n')
    interest_sectl = [x for x in interest_sectl if x.strip() != ""]
    comments_section = soups.find('div', id='comments-section').find('h2').find('span').find('a').text.strip().split(' ')[1]
    reviews = soups.find('section', id='reviews-wrapper').find('h2').find('span').find('a').text.strip().split(' ')[1]
    discussion = soups.find('div', class_='section-discussion').find('p',class_= 'pl').find('a').text.strip().split(' ')[-1]

    # print(interest_sectl)  # todo parse interest_sectl
    insert_data = {'_id': _id, 'name': name, 'year': year, 'infos': infos,
                   'douban_score': interest_sectl,'comments_section': comments_section,
                   'reviews': reviews, 'discussion': discussion}
    try:
        douban_movie_detail.insertOne(insert_data)
    except Exception as e:
        douban_movie_detail.updateOne({'_id': insert_data["_id"]}, insert_data)
    seed['movie_detail_status'] = 'success'
    douban_type_list.updateOne({'_id': seed["_id"]}, seed)
    douban_type_list.close()
    douban_movie_detail.close()
    return


def run_movie_detail():
    print('run_movie_detail')
    douban_type_list = mongo_manager('douban_type_list', db='douban')
    # douban_type_list.updateMany({}, {'movie_detail_status': '0'})
    # douban_type_list.close()
    while True:
        lists = []
        # for __ in range(100):
        #     seed = douban_type_list.find_one_and_update({'movie_detail_status':'0'},{'movie_detail_status':'2'})
        #     lists.append(seed)
        seeds = douban_type_list.findAll({'movie_detail_status': '0'}).limit(100)

        for seed in seeds:
            lists.append(seed)
        print('-' * 20, len(lists))
        pool = Pool(processes=min(5, len(lists) + 1))
        pool.map(save_movie_detail, lists)
        pool.close()  # 关闭进程池，不再接受新的进程
        pool.join()
        if len(lists) < 100:
            break
        break


if __name__ == '__main__':
    run_movie_detail()
